#!/bin/bash

########
## filename:Distcp.sh
## use distcp transport passlog
## hongxing.fan
########

dir=$(dirname $0)
cd $dir

## args
if [[ $# -eq 0 ]]
then
		cur_date=$(date +"%Y%m%d" -d "-1 day")
elif [[ $# -eq 1 ]]
then
	cur_date=$1
else
	echo "usage:sh Distcp.sh [20150606]"
	exit 1
fi

source ~/.bashrc
source /etc/profile

## which tpl appeared
HADOOP_BIN=/home/work/hadoop-client/hadoop/bin
SRC_PATH=/app/passport/pass_data/pass_data.db

REMOTE_PATH=hdfs://szwg-stoff-hdfs.dmop.baidu.com:54310/app/passport/pass_data/pass_data.db
LOCAL_PATH_59=webhdfs://nj03-game-m22dianquan59.nj03.baidu.com:8570/user/hive/warehouse/pass_data.db
LOCAL_PATH_58=webhdfs://nj03-game-m22dianquan58.nj03.baidu.com:8570/user/hive/warehouse/pass_data.db
LOCAL_PATH=$LOCAL_PATH_59


function isExist {
	if [[ $# -eq 1 ]]
	then
		tab_name=$1
		num=$($HADOOP_BIN/hadoop dfs -ls $SRC_PATH/$tab_name/dt=$cur_date | sort | uniq | wc -l)
		if [[ $num -gt 1 ]];then
			echo 0
		else
			echo 1
		fi
	elif [[ $# -eq 2 ]]
	then
		tab_name=$1
		tpl=$2
		num=$($HADOOP_BIN/hadoop dfs -ls $SRC_PATH/$tab_name/tpl=*/dt=$cur_date | awk -F" " '{print $8}' | awk -F"tpl=|/dt=" '{print $2}' | sort | uniq | wc -l)
		if [[ $num -gt 100 ]];then
			echo 0
		else
			echo 1
		fi
	else
		echo "usage:sh isExist tablename [tpl]"
		exit 1
	fi
}

function transport {
	if [[ $# -eq 1 ]]
	then
		tab_name=$1
		echo "****$REMOTE_PATH/$tab_name/dt=$cur_date<==>$LOCAL_PATH/$tab_name/dt=$cur_date*****"
		$HADOOP_BIN/hadoop distcp -update -i -m 800 $REMOTE_PATH/$tab_name/dt=$cur_date $LOCAL_PATH/$tab_name/dt=$cur_date &
	elif [[ $# -eq 2 ]]
	then
		tab_name=$1
		tpl=$2
		tpls=$($HADOOP_BIN/hadoop dfs -ls $SRC_PATH/$tab_name/tpl=*/dt=$cur_date | awk -F" " '{print $8}' | awk -F"tpl=|/dt=" '{print $2}' | sort | uniq)
		for t in ${tpls[*]}
		do
			num=$(ps aux | grep /home/work/hadoop-client/hadoop/bin/ | wc -l | cut -f1)
			num=$(expr $num - 1)
			while [[ $num -gt 20 ]]
			do
				sleep 10
				num=$(ps aux | grep /home/work/hadoop-client/hadoop/bin/ | wc -l | cut -f1)
				num=$(expr $num - 1)
			done
			echo "****$REMOTE_PATH/tpl=$t/dt=$cur_date<==>$LOCAL_PATH/tpl=$t/dt=$cur_date*****"
			{
				tryNum=0
				$HADOOP_BIN/hadoop distcp -update -i -m 800 $REMOTE_PATH/$tab_name/tpl=$t/dt=$cur_date $LOCAL_PATH/tpl=$t/dt=$cur_date
				stat=$?
				## 控制10个进程
				while [[ $stat -ne 0 && $tryNum -le 10 ]]
				do
					## 切换namenode
					tryNum=$(expr $tryNum + 1)
					if [[ `expr $tryNum % 2` -eq 0 ]];then
						LOCAL_PATH=$LOCAL_PATH_59
					else
						LOCAL_PATH=$LOCAL_PATH_58
					fi
					$HADOOP_BIN/hadoop distcp -update -i -m 800 $REMOTE_PATH/$tab_name/tpl=$t/dt=$cur_date $LOCAL_PATH/tpl=$t/dt=$cur_date
					stat=$?
				done
				echo "****$(date +'%Y-%m-%d %H:%M:%S')<==>path=$t/dt=$cur_date<==>status=$stat<==>tryNum=$tryNum****"
			}&
		done
		wait
	else
		echo "usage:sh transport tablename [tpl]"
		exit 1
	fi
}

TABLES=(ssn_log)
TABLES=(login_log reg_log ssn_log)
while [[ ${#TABLES[*]} -gt 0 ]]
 do
 	for tab in ${TABLES[*]}
 	do
 		case $tab in
 			login_log)
 				flag=`isExist "login_log"`
 				if [[ $flag -eq 0 ]];then
 					transport "login_log"
 					unset TABLES[0]
 				fi
 				;;
 			reg_log)
 				flag=`isExist "reg_log"`
 				if [[ $flag -eq 0 ]];then
 					transport "reg_log"
 					unset TABLES[1]
 				fi
 				;;
 			ssn_log)
 				flag=`isExist "ssn_log" "tpl"`
 				if [[ $flag -eq 0 ]];then
 					transport "ssn_log" "tpl"
 					unset TABLES[2]
 				fi
 				;;
 			*)
 				echo "not in table"
 				;;
 		esac
 
 	done
 	sleep $(expr 60 \* 10)
 done

## monitor
phone='13426082139 13811063282 13261676918 13520011291'
phone='13811063282'
host=$(hostname)
function sendMsg(){
	for number in $phone
	do
		gsmsend-script $number@$1 $2 /home/work/hadoop-client/hadoop/bin/Distcp.sh is error:"date=$cur_date"
	done
}

cmd=/home/hadoop/hadoop-2.5.0-cdh5.2.3/bin
reg_path=/user/hive/warehouse/pass_data.db/reg_log/dt=$cur_date
reg_num=$($cmd/hdfs dfs -ls $reg_path | wc -l)
if [[ $reg_num -le 2 ]];then
	sendMsg $host "reg_log"
fi
login_path=/user/hive/warehouse/pass_data.db/login_log/dt=$cur_date
login_num=$($cmd/hdfs dfs -ls $login_path | wc -l)
if [[ $login_num -le 4 ]];then
	sendMsg $host "login_log"
fi

ssn_path=/user/hive/warehouse/pass_data.db/ssn_log/*/dt=$cur_date
ssn_num=$($cmd/hdfs dfs -ls $ssn_path | wc -l)
if [[ $ssn_num -le 1300 ]];then
	sendMsg $host "ssn_log"
fi

